{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 准备工作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lxml.html import fromstring\n",
    "import time\n",
    "from random import random\n",
    "import requests_html\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "# encoding:utf-8\n",
    "from PIL import Image\n",
    "from selenium import webdriver\n",
    "import requests\n",
    "# import time\n",
    "import hashlib\n",
    "import base64\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 进入知网与文章搜索页"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from selenium import webdriver\n",
    "# from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "\n",
    "#caps=dict()\n",
    "#caps[\"pageLoadStrategy\"] = \"none\"   # Do not wait for full page load\n",
    "# 指定下载路径\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--incognito') # 无痕模式\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': 'D:\\\\爬虫\\\\知网文章-PDF'}\n",
    "opts.add_experimental_option('prefs', prefs)\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "#opts.add_argument('--headless') #浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败\n",
    "\n",
    "#opts.binary_location = r\"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe\" #\"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "\n",
    "# \"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "driver = webdriver.Chrome(options=opts) \n",
    "# driver.set_page_load_timeout(10)\n",
    "\n",
    "# while True:\n",
    "#     try:\n",
    "#         driver.get(url)\n",
    "#         break\n",
    "#     except:\n",
    "#         pass  \n",
    "# driver.maximize_window()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 加载时间过长\n",
    "driver.set_page_load_timeout(10)\n",
    "while True:\n",
    "    try:\n",
    "        driver.get(\"https://kns.cnki.net/kns8/defaultresult/index\")\n",
    "        break\n",
    "    except:\n",
    "        pass  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"https://kns.cnki.net/kns8/defaultresult/index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 检查登录状态\n",
    "element = driver.find_element_by_id('Ecp_loginShowName1')\n",
    "login = element.get_attribute('innerHTML')\n",
    "# 判断并IP登陆\n",
    "if login != '中山大学南方学院':\n",
    "    driver.find_element_by_xpath('//*[@id=\"Ecp_top_login\"]/a').click()\n",
    "    driver.find_element_by_xpath('//*[@id=\"Ecp_Button2\"]').click()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_id('txt_search').send_keys('VOSviewer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 搜索\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div[1]/input[2]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "高级检索\n"
     ]
    }
   ],
   "source": [
    "# 进入高级检索\n",
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/a[2]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-8-a8a363765572>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<span>学术期刊</span><em></em>\n"
     ]
    }
   ],
   "source": [
    "# 点击学术期刊\n",
    "element = driver.find_element_by_xpath('//ul[@class=\"doctype-menus keji\"]/li[@data-id=\"xsqk\"]/a')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "专业检索\n"
     ]
    }
   ],
   "source": [
    "# 选择专业检索\n",
    "element = driver.find_element_by_name('majorSearch')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 选择期刊来源\n",
    "\n",
    "## 北大核心来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"HX\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## EI来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"EI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## CSD来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"CSD\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## SCI来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"SI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## cssci来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"CSI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置搜索query\n",
    "# query = 'SU = \"新媒体\" AND (TI = \"人工智能\" OR TI = \"大数据\" OR TI = \"AI\" OR TI = \"big data\" )'\n",
    "query = 'SU = \"VOSviewer\" '\n",
    "element = driver.find_element_by_xpath('//textarea')\n",
    "element.clear()\n",
    "element.send_keys(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('//input[@value =\"检索\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导出refworks文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击每页显示文章数量\n",
    "time.sleep(5) #等一等\n",
    "driver.find_element_by_xpath('//*[@id=\"perPageDiv\"]/div').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择为50篇\n",
    "driver.find_element_by_xpath('//*[@id=\"perPageDiv\"]/div/i').click()\n",
    "time.sleep(3) \n",
    "driver.find_element_by_xpath('//*[@id=\"perPageDiv\"]/ul/li[3]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#勾选全选\n",
    "driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/label').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择“导出与分析”\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]').click()\n",
    "# 选择“导出文件”\n",
    "time.sleep(3) \n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]').click()\n",
    "# 选择Refworks格式\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-97-3e22ff9578d3>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[0])\n"
     ]
    }
   ],
   "source": [
    "# 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 导出txt\n",
    "driver.find_element_by_xpath('//*[@id=\"litotxt\"]/a').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动爬取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 先确定总页数，以便于规定循环翻页的次数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文章数量总计: 135\n",
      "结果页数总计: 3页\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import time\n",
    "from random import random\n",
    "import requests_html\n",
    "element = driver.find_element_by_xpath('//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "文章数量 = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')[0].lxml.text_content()\n",
    "print(\"文章数量总计:\",文章数量)\n",
    "结果页数 = html.xpath('//*[@id=\"gridTable\"]/div[2]/span[1]')[0].lxml.text_content()[1:3]\n",
    "print(\"结果页数总计:\",结果页数)\n",
    "\n",
    "pagenum = int(结果页数[0:1])\n",
    "print(pagenum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 清除\n",
    "driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 回到首页\n",
    "driver.find_element_by_xpath('//*[@id=\"total\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-134-704fdf3805c1>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "# 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "已选文献 = driver.find_element_by_xpath('//*[@id=\"selectCount\"]').get_attribute('innerHTML')\n",
    "int(已选文献)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "当前页数 = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[2]/span[2]').get_attribute('innerHTML')\n",
    "int(当前页数)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "当前页数 1\n",
      "50\n",
      "当前页数 2\n",
      "100\n",
      "当前页数 3\n",
      "已经全选所有页面文献\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-151-741b839f972a>:33: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "source": [
    "for page in range(1,pagenum+1):\n",
    "#     element = driver.find_element_by_xpath('//html')\n",
    "#     main_content =element.get_attribute('outerHTML')  \n",
    "#     html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "    已选文献 = driver.find_element_by_xpath('//*[@id=\"selectCount\"]').get_attribute('innerHTML')\n",
    "    print(int(已选文献))\n",
    "    当前页数 = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[2]/span[2]').get_attribute('innerHTML')\n",
    "    print(\"当前页数\",int(当前页数))\n",
    "    # Alert Text: 不能超过500个,若要重新选择,请按清除按钮,再进行选取操作\n",
    "    if int(当前页数) < pagenum:\n",
    "        # 勾选本页文献\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/label').click()\n",
    "        # 翻页\n",
    "#         element = driver.find_element_by_xpath('//html')\n",
    "#         main_content =element.get_attribute('outerHTML')  \n",
    "#         main_content\n",
    "        a = driver.find_element_by_xpath('//*[@id=\"PageNext\"]')\n",
    "        a.click()\n",
    "        # 等待时间 平均时间38*103\n",
    "        time.sleep(3+random()*8)\n",
    "    else:\n",
    "        print(\"已经全选所有页面文献\")\n",
    "        element = driver.find_element_by_xpath('//html')\n",
    "        main_content =element.get_attribute('outerHTML') \n",
    "        main_content\n",
    "        driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]').click()\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]').click()\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]').click()\n",
    "# ----------------------------进入导出页面------------------------------------\n",
    "        driver.switch_to_window(driver.window_handles[2])\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('/html/body/div[3]/div/div[1]/ul/li[8]').click()\n",
    "        driver.find_element_by_xpath('//*[@id=\"litotxt\"]').click()\n",
    "# ----------------------------清除已选文章，继续翻页------------------------------------\n",
    "        driver.switch_to.window(driver.window_handles[1])\n",
    "        element = driver.find_element_by_xpath('//html')\n",
    "        main_content =element.get_attribute('outerHTML') \n",
    "        a = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a').click()\n",
    "        time.sleep(3+random()*10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文章下载——PDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 点击谋篇文章标题\n",
    "driver.find_element_by_xpath('//*[@id=\"gridTable\"]/table/tbody/tr[8]/td[2]/a').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-15-0e0daf12b38e>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 点击pdf下载\n",
    "driver.find_element_by_xpath('//*[@id=\"pdfDown\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 若有验证码，则弹出新页面--->换窗口\n",
    "driver.switch_to_window(driver.window_handles[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 对验证码进行截图\n",
    "# encoding:utf-8\n",
    "from PIL import Image\n",
    "from selenium import webdriver\n",
    "import requests\n",
    "import time\n",
    "import hashlib\n",
    "import base64\n",
    "import json\n",
    " \n",
    "# url = 'https://weixin.sogou.com/antispider/?from=http%3A%2F%2Fweixin.sogou.com%2Fweixin%3Ftype%3D2%26query%3Dpython'\n",
    "# driver = webdriver.Chrome()\n",
    "# driver.maximize_window()  # 将浏览器最大化\n",
    "# driver.get(url)\n",
    "# # 截取当前网页并放到D盘下命名为printscreen，该网页有我们需要的验证码\n",
    "driver.save_screenshot('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\printscreen.png')\n",
    "imgelement = driver.find_element_by_id('vImg')  # 定位验证码\n",
    "location = imgelement.location  # 获取验证码x,y轴坐标\n",
    "print(location)\n",
    "size = imgelement.size  # 获取验证码的长宽\n",
    "print(size)\n",
    "rangle = (int(location['x']+260), int(location['y']+135), int(location['x'] + size['width']+360),\n",
    "          int(location['y'] + size['height']+170))  # 写成我们需要截取的位置坐标\n",
    "i = Image.open(\"C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\web_imgprintscreen.png\")  # 打开截图\n",
    "frame4 = i.crop(rangle)  # 使用Image的crop函数，从截图中再次截取我们需要的区域\n",
    "frame4=frame4.convert('RGB')\n",
    "frame4.save('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg') # 保存我们接下来的验证码图片 进行打码\n",
    " \n",
    "# driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 讯飞OCR识别验证码\n",
    "\"\"\"\n",
    "  印刷文字识别WebAPI接口调用示例接口文档(必看)：https://doc.xfyun.cn/rest_api/%E5%8D%B0%E5%88%B7%E6%96%87%E5%AD%97%E8%AF%86%E5%88%AB.html\n",
    "  上传图片base64编码后进行urlencode要求base64编码和urlencode后大小不超过4M最短边至少15px，最长边最大4096px支持jpg/png/bmp格式\n",
    "  (Very Important)创建完webapi应用添加合成服务之后一定要设置ip白名单，找到控制台--我的应用--设置ip白名单，如何设置参考：http://bbs.xfyun.cn/forum.php?mod=viewthread&tid=41891\n",
    "  错误码链接：https://www.xfyun.cn/document/error-code (code返回错误码时必看)\n",
    "  @author iflytek\n",
    "\"\"\"\n",
    "#-*- coding: utf-8 -*-\n",
    "import requests\n",
    "import time\n",
    "import hashlib\n",
    "import base64\n",
    "import json\n",
    "#from urllib import parse\n",
    "# 印刷文字识别 webapi 接口地址\n",
    "URL = \"http://webapi.xfyun.cn/v1/service/v1/ocr/general\"\n",
    "# 应用ID (必须为webapi类型应用，并印刷文字识别服务，参考帖子如何创建一个webapi应用：http://bbs.xfyun.cn/forum.php?mod=viewthread&tid=36481)\n",
    "APPID = \"600a7fb3\"\n",
    "# 接口密钥(webapi类型应用开通印刷文字识别服务后，控制台--我的应用---印刷文字识别---服务的apikey)\n",
    "API_KEY = \"73071c3d054a8bfc81f2ef13debafb91\"\n",
    "def getHeader():\n",
    "#  当前时间戳\n",
    "    curTime = str(int(time.time()))\n",
    "#  支持语言类型和是否开启位置定位(默认否)\n",
    "    param = {\"language\": \"cn|en\", \"location\": \"false\"}\n",
    "    param = json.dumps(param)\n",
    "    paramBase64 = base64.b64encode(param.encode('utf-8'))\n",
    "\n",
    "    m2 = hashlib.md5()\n",
    "    str1 = API_KEY + curTime + str(paramBase64,'utf-8')\n",
    "    m2.update(str1.encode('utf-8'))\n",
    "    checkSum = m2.hexdigest()\n",
    "# 组装http请求头\n",
    "    header = {\n",
    "        'X-CurTime': curTime,\n",
    "        'X-Param': paramBase64,\n",
    "        'X-Appid': APPID,\n",
    "        'X-CheckSum': checkSum,\n",
    "        'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',\n",
    "    }\n",
    "    return header\n",
    "#上传文件并进行base64位编码\n",
    "with open(r'C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg', 'rb') as f:\n",
    "    f1 = f.read()\n",
    "\n",
    "f1_base64 = str(base64.b64encode(f1), 'utf-8')\n",
    "\n",
    "# ## 网络外链转换base64位编码\n",
    "\n",
    "# f2_base64 = str(base64.b64encode(base64data), 'utf-8')\n",
    "\n",
    "\n",
    "data = {\n",
    "        'image':f1_base64\n",
    "        }\n",
    "\n",
    "\n",
    "\n",
    "r = requests.post(URL, data=data, headers=getHeader())\n",
    "result = str(r.content, 'utf-8')\n",
    "# # 错误码链接：https://www.xfyun.cn/document/error-code (code返回错误码时必看)\n",
    "# print(result)\n",
    "# input(\"Entry the any key to exit\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 观察返回值\n",
    "r.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 若有OCR返回值，得到验证码\n",
    "yzm = r.json()['data']['block'][0]['line'][0]['word'][0]['content'()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输入验证码\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('//*[@id=\"vcode\"]').send_keys(yzm)\n",
    "## 提交\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#准备工作\n",
    "import math\n",
    "import time\n",
    "from random import random\n",
    "import requests_html\n",
    "from PIL import Image\n",
    "from selenium import webdriver\n",
    "import requests\n",
    "import hashlib\n",
    "import base64\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-26-a8a363765572>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文章数量总计: 135\n",
      "结果页数总计: 7页\n",
      "836\n",
      "<700时 836\n",
      "xpath //*[@id=\"gridTable\"]/table/tbody/tr[19]/td[2]/a\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-24-427efd185265>:25: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n",
      "<ipython-input-24-427efd185265>:30: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ex_page 3\n",
      "hov_page 3\n",
      "关闭当前页\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-24-427efd185265>:163: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "836\n",
      "<700时 836\n",
      "xpath //*[@id=\"gridTable\"]/table/tbody/tr[20]/td[2]/a\n",
      "ex_page 3\n",
      "hov_page 3\n",
      "关闭当前页\n",
      "836\n",
      "<700时 836\n",
      "xpath //*[@id=\"gridTable\"]/table/tbody/tr[19]/td[2]/a\n",
      "ex_page 3\n",
      "hov_page 3\n",
      "关闭当前页\n",
      "836\n",
      "<700时 836\n",
      "xpath //*[@id=\"gridTable\"]/table/tbody/tr[20]/td[2]/a\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-24-427efd185265>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     22\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"xpath\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mxpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m         \u001b[1;31m## 切换窗口\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m         \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     25\u001b[0m         \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mswitch_to_window\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwindow_handles\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m         \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "element = driver.find_element_by_xpath('//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "文章数量 = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')[0].lxml.text_content()\n",
    "print(\"文章数量总计:\",文章数量)\n",
    "结果页数 = html.xpath('//*[@id=\"gridTable\"]/div[2]/span[1]')[0].lxml.text_content()[1:3]\n",
    "print(\"结果页数总计:\",结果页数)\n",
    "pagenum = int(结果页数[0:1])\n",
    "## 翻页\n",
    "for page in range(1,pagenum+1):\n",
    "    for i in range(1,21): ##50篇一页or20篇一页\n",
    "        time.sleep(10)\n",
    "        down_xpath = '//*[@id=\"gridTable\"]/table/tbody/tr['+str(i)+']/td[8]'\n",
    "        d = driver.find_element_by_xpath(down_xpath).get_attribute('innerHTML')\n",
    "        print(len(d))\n",
    "        if len(d)<700:\n",
    "            i += 1\n",
    "        else:\n",
    "            i = i\n",
    "        xpath = '//*[@id=\"gridTable\"]/table/tbody/tr['+str(i)+']/td[2]/a'\n",
    "        print(\"xpath\",xpath)\n",
    "        ## 切换窗口\n",
    "        time.sleep(3)\n",
    "        driver.switch_to_window(driver.window_handles[1])\n",
    "        time.sleep(5)\n",
    "        driver.find_element_by_xpath(xpath).click()\n",
    "        ## 切换窗口\n",
    "        time.sleep(8)\n",
    "        driver.switch_to_window(driver.window_handles[2])\n",
    "        ## 此时页面数量\n",
    "        ex_page = len(driver.window_handles)\n",
    "        print(\"ex_page\",ex_page)\n",
    "        \n",
    "        ## 点击pdf下载\n",
    "        time.sleep(8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"pdfDown\"]').click()\n",
    "        time.sleep(15)\n",
    "        ## 此时页面数量\n",
    "        hov_page = len(driver.window_handles)\n",
    "        print(\"hov_page\",hov_page)\n",
    "\n",
    "        if hov_page>ex_page:\n",
    "            ## 切换窗口\n",
    "            time.sleep(15)\n",
    "            driver.switch_to_window(driver.window_handles[3])\n",
    "            c = 0 \n",
    "            while c<1:            \n",
    "                # 开始验证码\n",
    "                ## 截图\n",
    "                # url = 'https://weixin.sogou.com/antispider/?from=http%3A%2F%2Fweixin.sogou.com%2Fweixin%3Ftype%3D2%26query%3Dpython'\n",
    "                # driver = webdriver.Chrome()\n",
    "                # driver.maximize_window()  # 将浏览器最大化\n",
    "                # driver.get(url)\n",
    "                # # 截取当前网页并放到D盘下命名为printscreen，该网页有我们需要的验证码\n",
    "                driver.save_screenshot('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\printscreen.png')\n",
    "                imgelement = driver.find_element_by_id('vImg')  # 定位验证码\n",
    "                location = imgelement.location  # 获取验证码x,y轴坐标\n",
    "                print(location)\n",
    "                size = imgelement.size  # 获取验证码的长宽\n",
    "                print(size)\n",
    "                rangle = (int(location['x']+260), int(location['y']+135), int(location['x'] + size['width']+360),\n",
    "                          int(location['y'] + size['height']+170))  # 写成我们需要截取的位置坐标\n",
    "                i = Image.open(\"C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\printscreen.png\")  # 打开截图\n",
    "                frame4 = i.crop(rangle)  # 使用Image的crop函数，从截图中再次截取我们需要的区域\n",
    "                frame4=frame4.convert('RGB')\n",
    "                frame4.save('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg') # 保存我们接下来的验证码图片 进行打码\n",
    "                # driver.close()\n",
    "                ## OCR\n",
    "                # 印刷文字识别 webapi 接口地址\n",
    "                print(\"开始OCR\")\n",
    "                URL = \"http://webapi.xfyun.cn/v1/service/v1/ocr/general\"\n",
    "                # 应用ID (必须为webapi类型应用，并印刷文字识别服务，参考帖子如何创建一个webapi应用：http://bbs.xfyun.cn/forum.php?mod=viewthread&tid=36481)\n",
    "                APPID = \"600a7fb3\"\n",
    "                # 接口密钥(webapi类型应用开通印刷文字识别服务后，控制台--我的应用---印刷文字识别---服务的apikey)\n",
    "                API_KEY = \"73071c3d054a8bfc81f2ef13debafb91\"\n",
    "                def getHeader():\n",
    "                #  当前时间戳\n",
    "                    curTime = str(int(time.time()))\n",
    "                #  支持语言类型和是否开启位置定位(默认否)\n",
    "                    param = {\"language\": \"cn|en\", \"location\": \"false\"}\n",
    "                    param = json.dumps(param)\n",
    "                    paramBase64 = base64.b64encode(param.encode('utf-8'))\n",
    "\n",
    "                    m2 = hashlib.md5()\n",
    "                    str1 = API_KEY + curTime + str(paramBase64,'utf-8')\n",
    "                    m2.update(str1.encode('utf-8'))\n",
    "                    checkSum = m2.hexdigest()\n",
    "                # 组装http请求头\n",
    "                    header = {\n",
    "                        'X-CurTime': curTime,\n",
    "                        'X-Param': paramBase64,\n",
    "                        'X-Appid': APPID,\n",
    "                        'X-CheckSum': checkSum,\n",
    "                        'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',\n",
    "                    }\n",
    "                    return header\n",
    "                #上传文件并进行base64位编码\n",
    "                with open(r'C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg', 'rb') as f:\n",
    "                    f1 = f.read()\n",
    "                f1_base64 = str(base64.b64encode(f1), 'utf-8')\n",
    "                # ## 网络外链转换base64位编码\n",
    "                # f2_base64 = str(base64.b64encode(base64data), 'utf-8')\n",
    "                data = {\n",
    "                        'image':f1_base64\n",
    "                        }\n",
    "\n",
    "                r = requests.post(URL, data=data, headers=getHeader())\n",
    "                # 判断是否识别出'content'\n",
    "                if len(r.json()['data']['block'][0]['line']) != 0:\n",
    "                    yzm = r.json()['data']['block'][0]['line'][0]['word'][0]['content']\n",
    "                    print(yzm)\n",
    "                    isalnum = yzm.isalnum()\n",
    "                    # 判断'content'是否只包含字母和数字\n",
    "                    if isalnum == True:\n",
    "                        ## 输入验证码\n",
    "                        time.sleep(8)\n",
    "                        driver.find_element_by_xpath('//*[@id=\"vcode\"]').send_keys(yzm)\n",
    "                        E = os.listdir(\"D:\\\\爬虫\\\\知网文章-PDF\")\n",
    "                        for e in E:\n",
    "                            e\n",
    "                        ex_nmb = len(e)\n",
    "                        print(\"ex_nmb\",ex_nmb)\n",
    "                        ## 提交\n",
    "                        print(\"准备提交\")\n",
    "                        time.sleep(8)\n",
    "                        driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click()\n",
    "                        print(\"已提交\")\n",
    "                        time.sleep(180) ## 网速慢，可能下载文件慢\n",
    "                        H = os.listdir(\"D:\\\\爬虫\\\\知网文章-PDF\")\n",
    "                        for h in H:\n",
    "                            h\n",
    "                        hov_nmb = len(h)\n",
    "                        print(\"hov_nmb\",hov_nmb)\n",
    "                        ## 验证码是否正确——>下载文章数是否增加\n",
    "                        if ex_nmb<hov_nmb:\n",
    "                            ## 关闭当前页面，返回首页下载下一篇\n",
    "                            driver.close()\n",
    "                            ## 切换窗口\n",
    "                            time.sleep(8)\n",
    "    #                         driver.switch_to_window(driver.window_handles[1])\n",
    "                            driver.close()\n",
    "                            c = 1\n",
    "                            i += 1\n",
    "                        else:\n",
    "                            c = 0\n",
    "                    else:\n",
    "\n",
    "                        time.sleep(8)\n",
    "                        driver.find_element_by_xpath('//*[@id=\"vImg\"]').click()\n",
    "                        c = 0\n",
    "    #                 else:\n",
    "    #                     print(\"换验证码\")\n",
    "    #             else:\n",
    "    #                 print(\"换验证码\")\n",
    "\n",
    "            else:           \n",
    "                driver.find_element_by_xpath('//*[@id=\"vImg\"]').click()\n",
    "                c = 0\n",
    "        else:\n",
    "            driver.close()\n",
    "            print(\"关闭当前页\")\n",
    "            driver.switch_to_window(driver.window_handles[1])\n",
    "            i += 1 \n",
    "        page += 1\n",
    "        driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
